#!/usr/bin/python
#coding=utf-8

import sys
import os
import rados
import threading
import commands
import re
import copy
import logging
import logging.config

import global_list
from command import *

#OSD请求类
class ReqOsd:
    #self.h=链接集群的handle
    def __init__(self,handle,dict,reqid,logger):
        self.h=handle
        self.act=dict['act']
        self.page=dict['page']
        self.pagesize=dict['pagesize']
        self.val=dict['val']
        self.item=dict['item']
        self.host=dict['host']
        self.reqid=int(reqid)
        self.logger=logger

    #线程处理函数
    def ThreadDealer(self):
        resp={}
        resp['act']=str(self.act)
        resp['type']='osd'
        resp['status']='running'
        if self.act == 'getstats' or self.act == 'getstatsbyregex':
            resp['page']=self.page
        else:
            resp['item']=self.item

        #tname=threading.currentThread().getName()
        mtx=global_list.get_value('MUTEX')
        #加锁->获取结果列表->追加结果->更新列表->解锁
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

        #执行具体命令
        if self.act == 'getstats':
            ret=self.getstats()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'getstatsbyregex':
            ret=self.getstatsbyregex()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'start':
            ret=self.start()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'stop':
            ret=self.stop()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'restart':
            ret=self.restart()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'getreweight':
            ret=self.getreweight()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'setreweight':
            ret=self.setreweight()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'getnearfull':
            ret=self.getnearfull()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'setnearfull':
            ret=self.setnearfull()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'getfull':
            ret=self.getfull()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'setfull':
            ret=self.setfull()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'setout':
            ret=self.setout()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        else:
            pass
 
        #更新结果
        mtx=global_list.get_value('MUTEX')
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

    #辅助函数，用于获取OSD总数
    def osd_num(self):
        d=dict()
        osd_stat = CephClusterCommand(self.h, prefix='osd stat', format='json')
        if osd_stat.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" ceph osd stat failed, error is "+str(osd_stat['err']))
            d['err']=osd_stat['err']
            return d
        result=osd_stat['result']
        d['count']=result.get('num_osds',0)
        return d
    
    #辅助函数，用于获取某个OSD的host和ip
    def find_osd(self, idx):
        d=dict()
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=idx, format='json')
        if osd_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd find failed, error is "+str(osd_find['err']))
            d['err']=osd_find['err']
            return d
        result=osd_find['result']
        d['host']=result.get('crush_location','None').get('host','None')
        d['ip']=result.get('ip','None')
        self.logger.info("reqid:"+str(self.reqid)+" osd id:"+str(idx)+" host:"+str(d['host'])+" ip:"+str(d['ip']))
        return d

    #获取集群osd信息
    def getstats(self):
        resplist=[]
        respdict=dict()

        #找到需要查询的OSD ID范围
        start_idx=int(self.pagesize)*(int(self.page)-1)
        end_idx=start_idx+int(self.pagesize)
        osd_ls = CephClusterCommand(self.h, prefix='osd ls', format='json')
        if osd_ls.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd ls failed, error is "+str(osd_ls['err']))
            respdict['err']=osd_ls['err']
            return respdict
        result=osd_ls['result']
        respdict['page']=int(self.page)
        respdict['count']=len(result)
        osdlist=result[start_idx:end_idx]

        #获取state和reweight
        osd_dump = CephClusterCommand(self.h, prefix='osd dump', format='json')
        if osd_dump.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd dump failed, error is "+str(osd_dump['err']))
            respdict['err']=osd_dump['err']
            return respdict
        osds=osd_dump['result'].get('osds', 'None')
        dumplist=[ x for x in osds if x['osd'] in osdlist ]

        #从osd df获取kb/kb_avail/utilization
        osd_df = CephClusterCommand(self.h, prefix='osd df', format='json')
        if osd_df.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd df failed, error is "+str(osd_df['err']))
            respdict['err']=osd_df['err']
            return respdict
        df=osd_df['result'].get('nodes')
        dflist=[ x for x in df if x['id'] in osdlist ]

        #采集信息
        for i in osdlist:
            d=dict()
            #从osd ls获取id/name
            d['id']=i
            d['name']="osd."+str(i)
            retval=self.find_osd(i)
            #从osd find获取host/ip
            if retval.has_key('err') == True:
                respdict['err']=retval['err']
                return respdict
            d['host']=retval['host']
            d['ip']=retval['ip']
            temp=[ x for x in dumplist if x['osd'] == i]
            #从osd dump获取reweight/state
            d['reweight']=temp[0].get('weight', 0)
            d['state']=[]
            if temp[0]['up'] == 1:
                d['state'].append('up')
            else:
                d['state'].append('down')
            if temp[0]['in'] == 1:
                d['state'].append('in')
            else:
                d['state'].append('out')

            #从osd df获取kb/kb_avail/utilization
            temp=[ x for x in dflist if x['id'] == i]
            d['kb']=temp[0].get('kb',0)
            d['kb_avail']=temp[0].get('kb_avail',0)
            d['utilization']=temp[0].get('utilization',0)

            #从ceph daemon获取nearfull
            cmd="/usr/bin/ssh "+d['host']+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(i)+".asok config get mon_osd_nearfull_ratio"
            ret,res=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" get mon_osd_nearfull_ratio by asok failed, ret is "+str(ret))
                respdict['err']='get config mon_osd_nearfull_ratio failed'
                return respdict
            d['nearfull_ratio']=float(res.replace("\n","").replace(" ","").replace("\"","").split(':')[1].split('}')[0])
            #从ceph daemon获取full
            cmd="/usr/bin/ssh "+d['host']+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(i)+".asok config get mon_osd_full_ratio"
            ret,res=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" get mon_osd_full_ratio by asok failed, ret is "+str(ret))
                respdict['err']='get config mon_osd_full_ratio failed'
                return respdict
            d['full_ratio']=float(res.replace("\n","").replace(" ","").replace("\"","").split(':')[1].split('}')[0])
            resplist.append(d)

        respdict['osds']=resplist
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #模糊查询集群osd信息
    def getstatsbyregex(self):
        resplist=[]
        respdict=dict()

        #找到需要查询的OSD ID范围
        start_idx=int(self.pagesize)*(int(self.page)-1)
        end_idx=start_idx+int(self.pagesize)
        osd_ls = CephClusterCommand(self.h, prefix='osd ls', format='json')
        if osd_ls.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd ls failed, error is "+str(osd_ls['err']))
            respdict['err']=osd_ls['err']
            return respdict
        result=osd_ls['result']

        #模糊查询OSD
        templist=[ x for x in result if None != re.search(str(self.val),"osd."+str(x))]
        if templist == []:
            self.logger.info("reqid:"+str(self.reqid)+" matching contents not found")
            respdict['osds']=[]
            return respdict
        respdict['page']=int(self.page)
        respdict['count']=len(templist)
        osdlist=templist[start_idx:end_idx]

        #获取state和reweight
        osd_dump = CephClusterCommand(self.h, prefix='osd dump', format='json')
        if osd_dump.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd dump failed, error is "+str(osd_dump['err']))
            respdict['err']=osd_dump['err']
            return respdict
        osds=osd_dump['result'].get('osds', 'None')
        dumplist=[ x for x in osds if x['osd'] in osdlist ]

        #采集信息
        for i in osdlist:
            d=dict()
            d['id']=i
            d['name']="osd."+str(i)
            retval=self.find_osd(i)
            if retval.has_key('err') == True:
                respdict['err']=retval['err']
                return respdict
            d['host']=retval['host']
            d['ip']=retval['ip']
            temp=[ x for x in dumplist if x['osd']==i]
            d['reweight']=temp[0].get('weight', 0)
            d['state']=[]
            if temp[0]['up'] == 1:
                d['state'].append('up')
            else:
                d['state'].append('down')
            if temp[0]['in'] == 1:
                d['state'].append('in')
            else:
                d['state'].append('out')

            #从ceph daemon获取nearfull
            cmd="/usr/bin/ssh "+d['host']+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(i)+".asok config get mon_osd_nearfull_ratio"
            ret,res=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" get mon_osd_nearfull_ratio by asok failed, ret is "+str(ret))
                respdict['err']='get config mon_osd_nearfull_ratio failed'
                return respdict
            d['nearfull']=float(res.replace("\n","").replace(" ","").replace("\"","").split(':')[1].split('}')[0])
            #从ceph daemon获取full
            cmd="/usr/bin/ssh "+d['host']+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(i)+".asok config get mon_osd_full_ratio"
            ret,res=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" get mon_osd_full_ratio by asok failed, ret is "+str(ret))
                respdict['err']='get config mon_osd_full_ratio failed'
                return respdict
            d['full']=float(res.replace("\n","").replace(" ","").replace("\"","").split(':')[1].split('}')[0])
            resplist.append(d)

        respdict['osds']=resplist
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #启动某个osd服务
    def start(self):
        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" find osd."+str(self.item)+" failed, error is "+osd_find['err'])
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+" at host: "+str(host))

        #利用systemctl，启动远程服务
        cmd="/usr/bin/systemctl -H "+str(host)+" start ceph-osd@"+str(self.item)+".service"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" start ceph-osd@"+str(self.item)+" failed, output: "+output)
            if output.find("systemctl reset-failed") != -1:
                self.logger.info("reqid:"+str(self.reqid)+" find reset-failed contents")
                cmd="/usr/bin/systemctl -H "+str(host)+" reset-failed ceph-osd@"+str(self.item)+".service"
                ret=os.system(cmd)
                if ret != 0:
                    self.logger.error("reqid:"+str(self.reqid)+" reset-failed failed")
                    return False
                else:
                    self.logger.info("reqid:"+str(self.reqid)+" reset-failed success")
                    cmd="/usr/bin/systemctl -H "+str(host)+" start ceph-osd@"+str(self.item)+".service"
                    ret=os.system(cmd)
                    if ret != 0:
                        self.logger.error("reqid:"+str(self.reqid)+" start ceph-osd@"+str(self.item)+" failed 2nd")
                        return False
                    self.logger.info("reqid:"+str(self.reqid)+" start ceph-osd@"+str(self.item)+" 2nd success")
            else:
                self.logger.error("reqid:"+str(self.reqid)+" no reset-failed found")
                return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" start ceph-osd@"+str(self.item)+" success")

        #检查服务是否启动成功
        cmd="/usr/bin/systemctl -H "+str(host)+" is-active ceph-osd@"+str(self.item)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res == 'active':
            self.logger.info("reqid:"+str(self.reqid)+" ceph-osd@"+str(self.item)+" is active now")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-osd@"+str(self.item)+" is-active failed, ret="+str(ret)+", res="+res)
            return False

    #停止某个osd服务
    def stop(self):
        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" find osd."+str(self.item)+" failed, error is "+osd_find['err'])
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+" at host: "+str(host))

        #利用systemctl，停止远程服务
        cmd="/usr/bin/systemctl -H "+str(host)+" stop ceph-osd@"+str(self.item)+".service"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" stop ceph-osd@"+str(self.item)+" failed, output: "+output)
            if output.find("systemctl reset-failed") != -1:
                self.logger.info("reqid:"+str(self.reqid)+" find reset-failed contents")
                cmd="/usr/bin/systemctl -H "+str(host)+" reset-failed ceph-osd@"+str(self.item)+".service"
                ret=os.system(cmd)
                if ret != 0:
                    self.logger.error("reqid:"+str(self.reqid)+" reset-failed failed")
                    return False
                else:
                    self.logger.info("reqid:"+str(self.reqid)+" reset-failed success")
                    cmd="/usr/bin/systemctl -H "+str(host)+" stop ceph-osd@"+str(self.item)+".service"
                    ret=os.system(cmd)
                    if ret != 0:
                        self.logger.error("reqid:"+str(self.reqid)+" stop ceph-osd@"+str(self.item)+" failed 2nd")
                        return False
                    self.logger.info("reqid:"+str(self.reqid)+" stop ceph-osd@"+str(self.item)+" 2nd success")
            else:
                self.logger.error("reqid:"+str(self.reqid)+" no reset-failed found")
                return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" stop ceph-osd@"+str(self.item)+" success")

        #检查服务是否停止成功
        cmd="/usr/bin/systemctl -H "+str(host)+" is-active ceph-osd@"+str(self.item)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res == 'inactive':
            self.logger.info("reqid:"+str(self.reqid)+" ceph-osd@"+str(self.item)+" is inactive now")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-osd@"+str(self.item)+" is-active failed, ret="+str(ret)+", res="+res)
            return False

    #重启某个osd服务
    def restart(self):
        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" find osd."+str(self.item)+" failed, error is "+osd_find['err'])
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+" at host: "+str(host))

        #利用systemctl，停止远程服务
        cmd="/usr/bin/systemctl -H "+str(host)+" restart ceph-osd@"+str(self.item)+".service"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" restart ceph-osd@"+str(self.item)+" failed, output: "+output)
            if output.find("systemctl reset-failed") != -1:
                self.logger.info("reqid:"+str(self.reqid)+" find reset-failed contents")
                cmd="/usr/bin/systemctl -H "+str(host)+" reset-failed ceph-osd@"+str(self.item)+".service"
                ret=os.system(cmd)
                if ret != 0:
                    self.logger.error("reqid:"+str(self.reqid)+" reset-failed failed")
                    return False
                else:
                    self.logger.info("reqid:"+str(self.reqid)+" reset-failed success")
                    cmd="/usr/bin/systemctl -H "+str(host)+" restart ceph-osd@"+str(self.item)+".service"
                    ret=os.system(cmd)
                    if ret != 0:
                        self.logger.error("reqid:"+str(self.reqid)+" restart ceph-osd@"+str(self.item)+" failed 2nd")
                        return False
                    self.logger.info("reqid:"+str(self.reqid)+" restart ceph-osd@"+str(self.item)+" 2nd success")
            else:
                self.logger.error("reqid:"+str(self.reqid)+" no reset-failed found") 
                return False

        #检查服务是否重启动成功
        cmd="/usr/bin/systemctl -H "+str(host)+" is-active ceph-osd@"+str(self.item)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res == 'active':
            self.logger.info("reqid:"+str(self.reqid)+" ceph-osd@"+str(self.item)+" is active now")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-osd@"+str(self.item)+" is-active failed, ret="+str(ret)+", res="+res)
            return False

    #获取reweight值
    def getreweight(self):
        respdict=dict()

        cmd="/usr/bin/ceph osd tree | /usr/bin/grep -w osd."+str(self.item)+" | awk '{print $5}'"
        ret,res=commands.getstatusoutput(cmd)
        if ret == 0:
            respdict['reweight']=float(res)
            self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+" reweight: "+str(res))
            return respdict
        else:
            self.logger.error("reqid:"+str(self.reqid)+" get osd."+str(self.item)+" reweight failed")
            respdict['err']='get reweight of osd failed'
            return respdict

    #设置reweight值
    def setreweight(self):
        cmd="/usr/bin/ceph osd reweight "+str(self.item)+" "+str(self.val)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" set osd."+str(self.item)+" reweight: "+str(self.val))
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" set osd."+str(self.item)+" reweight failed")
            return False

    #获取nearfull_ratio
    def getnearfull(self):
        respdict=dict()

        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            respdict['err']=osd_find['err']
            return respdict
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        if host == 'None':
            respdict['err']='get osd host failed'
            return respdict

        #利用ceph daemon获取某个OSD的nearfull值
        cmd="/usr/bin/ssh "+host+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(self.item)+".asok config get mon_osd_nearfull_ratio -f json"
        ret,res=commands.getstatusoutput(cmd)
        if ret == 0:
            val=json.loads(res)
            respdict['nearfull']=float(val['mon_osd_nearfull_ratio'])
            return respdict
        else:
            respdict['err']='get mon_osd_nearfull_ratio failed'
            return respdict
  
    #设置nearfull_ratio
    def setnearfull(self):
        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        if host == 'None':
            return False

        #利用ceph daemon设置某个OSD的nearfull值
        cmd="/usr/bin/ssh "+host+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(self.item)+\
                ".asok config set mon_osd_nearfull_ratio "+str(self.val)
        ret=os.system(cmd)
        if ret == 0:
            return True
        else:
            return False

    #获取full_ratio
    def getfull(self):
        respdict=dict()

        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            respdict['err']=osd_find['err']
            return respdict
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        if host == 'None':
            respdict['err']='get osd host failed'
            return respdict

        #利用ceph daemon获取某个OSD的full值
        cmd="/usr/bin/ssh "+host+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(self.item)+".asok config get mon_osd_full_ratio -f json"
        ret,res=commands.getstatusoutput(cmd)
        if ret == 0:
            val=json.loads(res)
            respdict['full']=float(val['mon_osd_full_ratio'])
            return respdict
        else:
            respdict['err']='get mon_osd_full_ratio failed'
            return respdict
    
    #设置full_ratio
    def setfull(self):
        #查找OSD所在主机
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=int(self.item), format='json')
        if osd_find.has_key('err') == True:
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
        if host == 'None':
            return False

        #利用ceph daemon设置某个OSD的full值
        cmd="/usr/bin/ssh "+host+" /usr/bin/ceph daemon /var/run/ceph/ceph-osd."+str(self.item)+\
                ".asok config set mon_osd_full_ratio "+str(self.val)
        ret=os.system(cmd)
        if ret == 0:
            return True
        else:
            return False

    #将某个OSD标记为out
    def setout(self):
        cmd='/usr/bin/ceph osd out '+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" marked osd."+str(self.item)+" out")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" marked osd."+str(self.item)+" out failed")
            return False

    #析构
    def __del__(self):
        pass
